package day02;

import org.apache.spark.SparkConf;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import scala.Tuple2;

import java.util.Arrays;

public class JavaWordCountLamda {
    public static void main(String[] args) {
        SparkConf conf = new SparkConf().setAppName("JavaWordCountLamda").setMaster("local[2]");
        JavaSparkContext jsc = new JavaSparkContext(conf);

        JavaRDD<String> lines = jsc.textFile("hdfs://master:9000/Wordcount3/input/one.txt");
        JavaRDD<String> words = lines.flatMap(x -> Arrays.asList(x.split(" ")).iterator());
        JavaPairRDD<String, Integer> tup = words.mapToPair(x -> new Tuple2<>(x, 1));
        JavaPairRDD<String, Integer> aggr = tup.reduceByKey((x1, x2) -> x1 + x2);
        JavaPairRDD<Integer, String> swaped = aggr.mapToPair(x -> x.swap());
        JavaPairRDD<Integer, String> sorted = swaped.sortByKey(false);
        JavaPairRDD<String, Integer> res = sorted.mapToPair(x -> x.swap());
        lines.partitions().size();
        System.out.println(res.collect());
        jsc.stop();

    }
}
